{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "502b2572",
   "metadata": {},
   "outputs": [],
   "source": [
    "import io\n",
    "import json\n",
    "\n",
    "from pathlib import Path\n",
    "from urllib.parse import urlparse, parse_qs\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "import requests\n",
    "import clipboard\n",
    "\n",
    "from PIL import Image\n",
    "from jinja2 import Template"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "189aae14",
   "metadata": {},
   "outputs": [],
   "source": [
    "module_number = 3\n",
    "module_title = 'Orchestration and ML Pipelines'\n",
    "module_folder = Path('..') / '03-orchestration'\n",
    "\n",
    "meta_json_file = module_folder / 'meta.json'\n",
    "playlist_id = 'PL3MmuxUbc_hIUISrluw_A7wDSmfOhErJK'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "325223ed",
   "metadata": {},
   "outputs": [],
   "source": [
    "module_number = 5\n",
    "module_title = 'ML Monitoring'\n",
    "module_folder = Path('..') / '05-monitoring'\n",
    "\n",
    "meta_json_file = module_folder / 'meta.json'\n",
    "playlist_id = 'PL3MmuxUbc_hIUISrluw_A7wDSmfOhErJK'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "6488e6e6",
   "metadata": {},
   "outputs": [],
   "source": [
    "module_number = 6\n",
    "module_title = 'Best Practices'\n",
    "module_folder = Path('..') / '06-best-practices'\n",
    "\n",
    "meta_json_file = module_folder / 'meta.json'\n",
    "playlist_id = 'PL3MmuxUbc_hIUISrluw_A7wDSmfOhErJK'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "921396f5",
   "metadata": {},
   "outputs": [],
   "source": [
    "module_number = 4\n",
    "module_title = 'Model Deployment'\n",
    "module_folder = Path('..') / '04-deployment'\n",
    "\n",
    "meta_json_file = module_folder / 'meta.json'\n",
    "playlist_id = 'PL3MmuxUbc_hIUISrluw_A7wDSmfOhErJK'"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a91381d0",
   "metadata": {},
   "source": [
    "## Prepare `meta.json` info"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "15d0782b",
   "metadata": {},
   "outputs": [],
   "source": [
    "text = \"\"\"\n",
    "module_number\tmodule_title\tunit_number\tunit_title\tfull_title\tyoutube\n",
    "5\tMonitoring\t1\tIntro to ML monitoring\tMLOps Zoomcamp 5.1 - Intro to ML monitoring\thttps://www.youtube.com/watch?v=SQ0jBwd_3kk\n",
    "5\tMonitoring\t2\tEnvironment setup\tMLOps Zoomcamp 5.2 - Environment setup\thttps://www.youtube.com/watch?v=yixA3C1xSxc\n",
    "5\tMonitoring\t3\tPrepare reference and model\tMLOps Zoomcamp 5.3 - Prepare reference and model\thttps://www.youtube.com/watch?v=IjNrkqMYQeQ\n",
    "5\tMonitoring\t4\tEvidently metrics calculation\tMLOps Zoomcamp 5.4 - Evidently metrics calculation\thttps://www.youtube.com/watch?v=kP3lzh_HfWY\n",
    "5\tMonitoring\t5\tDummy monitoring\tMLOps Zoomcamp 5.5 - Dummy monitoring\thttps://www.youtube.com/watch?v=s3G4PMsOMOA\n",
    "5\tMonitoring\t6\tData quality monitoring\tMLOps Zoomcamp 5.6 - Data quality monitoring\thttps://www.youtube.com/watch?v=fytrmPbcLhI\n",
    "5\tMonitoring\t7\tSave Grafana Dashboard\tMLOps Zoomcamp 5.7 - Save Grafana Dashboard\thttps://www.youtube.com/watch?v=-c4iumyZMyw\n",
    "5\tMonitoring\t8\tDebugging with test suites and reports\tMLOps Zoomcamp 5.8 - Debugging with test suites and reports\thttps://www.youtube.com/watch?v=sNSk3ojISh8\"\"\".strip()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "43db9fdf",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv(io.StringIO(text), delimiter='\\t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "38ffbb71",
   "metadata": {},
   "outputs": [],
   "source": [
    "df['youtube'] = df['youtube'].fillna('')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "aa4e2fb9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'number': 1,\n",
       "  'title': 'Intro to ML monitoring',\n",
       "  'youtube': 'https://www.youtube.com/watch?v=SQ0jBwd_3kk'},\n",
       " {'number': 2,\n",
       "  'title': 'Environment setup',\n",
       "  'youtube': 'https://www.youtube.com/watch?v=yixA3C1xSxc'},\n",
       " {'number': 3,\n",
       "  'title': 'Prepare reference and model',\n",
       "  'youtube': 'https://www.youtube.com/watch?v=IjNrkqMYQeQ'},\n",
       " {'number': 4,\n",
       "  'title': 'Evidently metrics calculation',\n",
       "  'youtube': 'https://www.youtube.com/watch?v=kP3lzh_HfWY'},\n",
       " {'number': 5,\n",
       "  'title': 'Dummy monitoring',\n",
       "  'youtube': 'https://www.youtube.com/watch?v=s3G4PMsOMOA'},\n",
       " {'number': 6,\n",
       "  'title': 'Data quality monitoring',\n",
       "  'youtube': 'https://www.youtube.com/watch?v=fytrmPbcLhI'},\n",
       " {'number': 7,\n",
       "  'title': 'Save Grafana Dashboard',\n",
       "  'youtube': 'https://www.youtube.com/watch?v=-c4iumyZMyw'},\n",
       " {'number': 8,\n",
       "  'title': 'Debugging with test suites and reports',\n",
       "  'youtube': 'https://www.youtube.com/watch?v=sNSk3ojISh8'}]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_units = df[['unit_number', 'unit_title', 'youtube']]\n",
    "units = df_units \\\n",
    "    .rename(columns={'unit_number': 'number', 'unit_title': 'title'}) \\\n",
    "    .to_dict(orient='records')\n",
    "units"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "295d55b6",
   "metadata": {},
   "outputs": [],
   "source": [
    "meta = {\n",
    "    'module': {\n",
    "        'number': module_number,\n",
    "        'title': module_title\n",
    "    },\n",
    "    'units': units\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "cf6ed0e9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'module': {'number': 5, 'title': 'ML Monitoring'},\n",
       " 'units': [{'number': 1,\n",
       "   'title': 'Intro to ML monitoring',\n",
       "   'youtube': 'https://www.youtube.com/watch?v=SQ0jBwd_3kk'},\n",
       "  {'number': 2,\n",
       "   'title': 'Environment setup',\n",
       "   'youtube': 'https://www.youtube.com/watch?v=yixA3C1xSxc'},\n",
       "  {'number': 3,\n",
       "   'title': 'Prepare reference and model',\n",
       "   'youtube': 'https://www.youtube.com/watch?v=IjNrkqMYQeQ'},\n",
       "  {'number': 4,\n",
       "   'title': 'Evidently metrics calculation',\n",
       "   'youtube': 'https://www.youtube.com/watch?v=kP3lzh_HfWY'},\n",
       "  {'number': 5,\n",
       "   'title': 'Dummy monitoring',\n",
       "   'youtube': 'https://www.youtube.com/watch?v=s3G4PMsOMOA'},\n",
       "  {'number': 6,\n",
       "   'title': 'Data quality monitoring',\n",
       "   'youtube': 'https://www.youtube.com/watch?v=fytrmPbcLhI'},\n",
       "  {'number': 7,\n",
       "   'title': 'Save Grafana Dashboard',\n",
       "   'youtube': 'https://www.youtube.com/watch?v=-c4iumyZMyw'},\n",
       "  {'number': 8,\n",
       "   'title': 'Debugging with test suites and reports',\n",
       "   'youtube': 'https://www.youtube.com/watch?v=sNSk3ojISh8'}]}"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "meta"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "e0197e13",
   "metadata": {},
   "outputs": [],
   "source": [
    "module_folder.mkdir(parents=True, exist_ok=True)\n",
    "\n",
    "with open(meta_json_file, 'wt') as f_out:\n",
    "    json.dump(meta, f_out, indent=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "a06089d0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "  \"module\": {\n",
      "    \"number\": 5,\n",
      "    \"title\": \"ML Monitoring\"\n",
      "  },\n",
      "  \"units\": [\n",
      "    {\n",
      "      \"number\": 1,\n",
      "      \"title\": \"Intro to ML monitoring\",\n",
      "      \"youtube\": \"https://www.youtube.com/watch?v=SQ0jBwd_3kk\"\n"
     ]
    }
   ],
   "source": [
    "!head {meta_json_file}"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5db87368",
   "metadata": {},
   "source": [
    "## Generate page"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "138aa0c4",
   "metadata": {},
   "source": [
    "module_folder = Path('..') / '05-monitoring'\n",
    "meta_json_file = module_folder / 'meta.json'\n",
    "playlist_id = 'PL3MmuxUbc_hIUISrluw_A7wDSmfOhErJK'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "166dadc7",
   "metadata": {},
   "outputs": [],
   "source": [
    "with meta_json_file.open('rt') as f_in:\n",
    "    meta = json.load(f_in)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "50fc8f7c",
   "metadata": {},
   "outputs": [],
   "source": [
    "module_info = meta['module']\n",
    "units = meta['units']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "32877ffd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'number': 5, 'title': 'ML Monitoring'}"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "module_info"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "a6340d93",
   "metadata": {},
   "outputs": [],
   "source": [
    "images_folder = module_folder / 'images'\n",
    "images_folder.mkdir(parents=True, exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "d92031f3",
   "metadata": {},
   "outputs": [],
   "source": [
    "template_string = \"\"\"\n",
    "## {{ module_number }}.{{ unit_number }} {{ unit_title }}\n",
    "\n",
    "{% if youtube %}<a href=\"{{ youtube }}\">\n",
    "  <img src=\"{{ thumbnail }}\">\n",
    "</a>{% endif %}{% if not youtube %}COMING SOON{% endif %}\n",
    "\n",
    "\n",
    "\"\"\".lstrip()\n",
    "\n",
    "template = Template(template_string)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "a87656f0",
   "metadata": {},
   "outputs": [],
   "source": [
    "def download_thumbnail(video, module, unit, folder):\n",
    "    if type(unit) in [int, np.int64]:\n",
    "        thumbnail_file = f'thumbnail-{module}-{unit:02d}.jpg'\n",
    "    else:\n",
    "        thumbnail_file = f'thumbnail-{module}-{unit}.jpg'\n",
    "\n",
    "    thumbnail_file = folder / thumbnail_file\n",
    "\n",
    "    if thumbnail_file.exists():\n",
    "        print(f'{thumbnail_file} exists')\n",
    "        return thumbnail_file\n",
    "\n",
    "    video_id = parse_qs(urlparse(video).query)['v'][0]\n",
    "    print(f'processing video {video_id}...')\n",
    "    thumbnail_url = f'https://img.youtube.com/vi/{video_id}/0.jpg'\n",
    "\n",
    "    response = requests.get(thumbnail_url)\n",
    "    thumbnail = Image.open(io.BytesIO(response.content))\n",
    "    w_img, h_img = thumbnail.size\n",
    "\n",
    "    play = Image.open(Path('../images/play.png'))\n",
    "    w_play, h_play = play.size\n",
    "    \n",
    "    x0 = w_img // 2 - w_play // 2\n",
    "    y0 = h_img // 2 - h_play // 2\n",
    "\n",
    "    thumbnail.paste(play, (x0, y0), play)\n",
    "    thumbnail.save(thumbnail_file, quality=90)\n",
    "\n",
    "    print('saved to', thumbnail_file)\n",
    "\n",
    "    return thumbnail_file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8910fbf3",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "b592beb6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing video SQ0jBwd_3kk...\n",
      "saved to ..\\05-monitoring\\images\\thumbnail-5-01.jpg\n",
      "## 5.1 Intro to ML monitoring\n",
      "\n",
      "<a href=\"https://www.youtube.com/watch?v=SQ0jBwd_3kk&list=PL3MmuxUbc_hIUISrluw_A7wDSmfOhErJK\">\n",
      "  <img src=\"images/thumbnail-5-01.jpg\">\n",
      "</a>\n",
      "\n",
      "\n",
      "processing video yixA3C1xSxc...\n",
      "saved to ..\\05-monitoring\\images\\thumbnail-5-02.jpg\n",
      "## 5.2 Environment setup\n",
      "\n",
      "<a href=\"https://www.youtube.com/watch?v=yixA3C1xSxc&list=PL3MmuxUbc_hIUISrluw_A7wDSmfOhErJK\">\n",
      "  <img src=\"images/thumbnail-5-02.jpg\">\n",
      "</a>\n",
      "\n",
      "\n",
      "processing video IjNrkqMYQeQ...\n",
      "saved to ..\\05-monitoring\\images\\thumbnail-5-03.jpg\n",
      "## 5.3 Prepare reference and model\n",
      "\n",
      "<a href=\"https://www.youtube.com/watch?v=IjNrkqMYQeQ&list=PL3MmuxUbc_hIUISrluw_A7wDSmfOhErJK\">\n",
      "  <img src=\"images/thumbnail-5-03.jpg\">\n",
      "</a>\n",
      "\n",
      "\n",
      "processing video kP3lzh_HfWY...\n",
      "saved to ..\\05-monitoring\\images\\thumbnail-5-04.jpg\n",
      "## 5.4 Evidently metrics calculation\n",
      "\n",
      "<a href=\"https://www.youtube.com/watch?v=kP3lzh_HfWY&list=PL3MmuxUbc_hIUISrluw_A7wDSmfOhErJK\">\n",
      "  <img src=\"images/thumbnail-5-04.jpg\">\n",
      "</a>\n",
      "\n",
      "\n",
      "processing video s3G4PMsOMOA...\n",
      "saved to ..\\05-monitoring\\images\\thumbnail-5-05.jpg\n",
      "## 5.5 Dummy monitoring\n",
      "\n",
      "<a href=\"https://www.youtube.com/watch?v=s3G4PMsOMOA&list=PL3MmuxUbc_hIUISrluw_A7wDSmfOhErJK\">\n",
      "  <img src=\"images/thumbnail-5-05.jpg\">\n",
      "</a>\n",
      "\n",
      "\n",
      "processing video fytrmPbcLhI...\n",
      "saved to ..\\05-monitoring\\images\\thumbnail-5-06.jpg\n",
      "## 5.6 Data quality monitoring\n",
      "\n",
      "<a href=\"https://www.youtube.com/watch?v=fytrmPbcLhI&list=PL3MmuxUbc_hIUISrluw_A7wDSmfOhErJK\">\n",
      "  <img src=\"images/thumbnail-5-06.jpg\">\n",
      "</a>\n",
      "\n",
      "\n",
      "processing video -c4iumyZMyw...\n",
      "saved to ..\\05-monitoring\\images\\thumbnail-5-07.jpg\n",
      "## 5.7 Save Grafana Dashboard\n",
      "\n",
      "<a href=\"https://www.youtube.com/watch?v=-c4iumyZMyw&list=PL3MmuxUbc_hIUISrluw_A7wDSmfOhErJK\">\n",
      "  <img src=\"images/thumbnail-5-07.jpg\">\n",
      "</a>\n",
      "\n",
      "\n",
      "processing video sNSk3ojISh8...\n",
      "saved to ..\\05-monitoring\\images\\thumbnail-5-08.jpg\n",
      "## 5.8 Debugging with test suites and reports\n",
      "\n",
      "<a href=\"https://www.youtube.com/watch?v=sNSk3ojISh8&list=PL3MmuxUbc_hIUISrluw_A7wDSmfOhErJK\">\n",
      "  <img src=\"images/thumbnail-5-08.jpg\">\n",
      "</a>\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "module_number = module_info['number']\n",
    "\n",
    "parts = []\n",
    "\n",
    "for unit in units:\n",
    "    unit_number = unit['number']\n",
    "\n",
    "    params = {\n",
    "        'module_number': module_info['number'],\n",
    "        'module_name': module_info['title'],\n",
    "        'unit_number': unit['number'],\n",
    "        'unit_title': unit['title']          \n",
    "    }\n",
    "\n",
    "    if 'youtube' in unit:\n",
    "        youtube = unit['youtube']\n",
    "        if len(youtube) and youtube.startswith('https'):\n",
    "            thumbnail = download_thumbnail(youtube, module_number, unit_number, images_folder)\n",
    "            thumbnail_path = '/'.join(thumbnail.parts[2:])\n",
    "            params['youtube'] = f'{youtube}&list={playlist_id}'\n",
    "            params['thumbnail'] = thumbnail_path\n",
    "        \n",
    "    template_string = template.render(params)\n",
    "    print(template_string)\n",
    "    parts.append(template_string)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "6d6e3584",
   "metadata": {},
   "outputs": [],
   "source": [
    "prefix = f\"\"\"\n",
    "# {module_info['number']}. {module_info['title']} \n",
    "\"\"\".strip()\n",
    "\n",
    "\n",
    "final_result = '\\n\\n'.join([prefix] + parts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "4d900666",
   "metadata": {},
   "outputs": [],
   "source": [
    "clipboard.copy(final_result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6ff9d662",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
